# # -*- coding: utf-8 -*-
# from scrapy import Request
# from scrapy.exceptions import IgnoreRequest
# from zc_core.dao.item_data_dao import ItemDataDao
# from zc_core.util.batch_gen import time_to_batch_no
# from zc_core.util.http_util import retry_request
# from sdszfcg.rules import *
#
#
# class GroupSpider(BaseSpider):
#     name = 'group'
#     # 常用链接
#     group_url = "http://ggzyjyzx.shandong.gov.cn/wssc/goods/{}/aggregation?from=mall&pid={}"
#
#     def __init__(self, batchNo=None, *args, **kwargs):
#         super(GroupSpider, self).__init__(*args, **kwargs)
#         if not batchNo:
#             self.batch_no = time_to_batch_no(datetime.now())
#         else:
#             self.batch_no = int(batchNo)
#
#     def start_requests(self):
#         pool_list = ItemDataDao().get_batch_data_list(self.batch_no, query={'spuId': {'$exists': False}})
#         self.logger.info('同款商品数量: %s' % (len(pool_list)))
#         # random.shuffle(pool_list)
#         for sku in pool_list:
#             sku_id = sku.get('_id')
#             group_url = self.group_url.format(sku_id, sku_id)
#             # 同款分组
#             yield Request(
#                 url=group_url,
#                 callback=self.parse_group,
#                 errback=self.error_back,
#                 meta={
#                     'reqType': 'group',
#                     'skuId': sku_id,
#                     'batchNo': self.batch_no,
#                 },
#                 headers={
#                     'siteCode': 'sdszfcg'
#                 },
#                 priority=5,
#             )
#
#     # 处理同款商品
#     def parse_group(self, response):
#         group = parse_group(response)
#         if group and group.get('skuIdList'):
#             self.logger.info('同款: sku=%s, cnt=%s' % (group.get('skuId'), len(group.get('skuIdList'))))
#             yield group
#
#     def error_back(self, e):
#         if e.type and e.type == IgnoreRequest:
#             self.logger.info(e.value)
#         else:
#             if e.request:
#                 self.logger.error('请求异常: [%s][%s] -> [%s]' % (str(type(e)), e.request.url, e.request.meta))
#                 yield retry_request(e.request)
#             else:
#                 self.logger.error('未知异常: %s' % e)
